 {
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First ensemble with NSL-KDD\n",
    "# Parameters\n",
    "\n",
    "#----------------------------------------------\n",
    "# 0 for not using it as base learner\n",
    "# 1 for using it as base learner\n",
    "\n",
    "use_model_ada = 1 \n",
    "use_model_dnn = 1 \n",
    "use_model_mlp = 1 \n",
    "use_model_lgbm = 1 \n",
    "use_model_rf = 1 \n",
    "use_model_svm = 1\n",
    "use_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "# 0 for training the model\n",
    "# 1 for using the saved version of the model\n",
    "\n",
    "load_model_ada = 0 \n",
    "load_model_dnn = 0 \n",
    "load_model_mlp = 0 \n",
    "load_model_lgbm = 0 \n",
    "load_model_rf = 0 \n",
    "load_model_svm = 0\n",
    "load_model_knn = 0 \n",
    "#----------------------------------------------\n",
    "\n",
    "# load_model_ada = 1\n",
    "# load_model_dnn = 1 \n",
    "# load_model_mlp = 1 \n",
    "# load_model_lgbm = 1 \n",
    "# load_model_rf = 1 \n",
    "# load_model_svm = 1\n",
    "# load_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "# feature_selection_bit = 0\n",
    "feature_selection_bit = 1\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Specify the name of the output text file\n",
    "if feature_selection_bit == 0:\n",
    "    output_file_name = \"ensemble_base_models_all_features.txt\"\n",
    "    with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---- ensemble_base_models_all_features', file = f)\n",
    "\n",
    "elif feature_selection_bit == 1:\n",
    "    output_file_name = \"ensemble_base_models_feature_selection.txt\"\n",
    "    with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('----ensemble_base_models_feature_selection--', file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle # saving and loading trained model\n",
    "from os import path\n",
    "\n",
    "\n",
    "# importing required libraries for normalizing data\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder,LabelEncoder, MinMaxScaler, OneHotEncoder)\n",
    "from sklearn.preprocessing import Normalizer, MaxAbsScaler , RobustScaler, PowerTransformer\n",
    "\n",
    "# importing library for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score # for calculating accuracy of model\n",
    "from sklearn.model_selection import train_test_split # for splitting the dataset for training and testing\n",
    "from sklearn.metrics import classification_report # for generating a classification report of model\n",
    "\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Dense # importing dense layer\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "# representation of model layers\n",
    "#from keras.utils import plot_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import shap\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# In[2]:\n",
    "\n",
    "\n",
    "# #Defining metric functions\n",
    "# def ACC(TP,TN,FP,FN):\n",
    "#     Acc = (TP+TN)/(TP+FP+FN+TN)\n",
    "#     return Acc\n",
    "# def ACC_2 (TP, FN):\n",
    "#     ac = (TP/(TP+FN))\n",
    "#     return ac\n",
    "# def PRECISION(TP,FP):\n",
    "#     eps = 1e-7\n",
    "#     Precision = TP/(TP+FP+eps)\n",
    "    \n",
    "\n",
    "#     return Precision\n",
    "# def RECALL(TP,FN):\n",
    "#     Recall = TP/(TP+FN)\n",
    "#     return Recall\n",
    "# def F1(Recall, Precision):\n",
    "#     F1 = 2 * Recall * Precision / (Recall + Precision)\n",
    "#     return F1\n",
    "# def BACC(TP,TN,FP,FN):\n",
    "#     BACC =(TP/(TP+FN)+ TN/(TN+FP))*0.5\n",
    "#     return BACC\n",
    "# def MCC(TP,TN,FP,FN):\n",
    "#     eps = 1e-7\n",
    "#     MCC = (TN*TP-FN*FP)/(((TP+FP+eps)*(TP+FN+eps)*(TN+FP+eps)*(TN+FN+eps))**.5)\n",
    "#     return MCC\n",
    "# def AUC_ROC(y_test_bin,y_score):\n",
    "#     fpr = dict()\n",
    "#     tpr = dict()\n",
    "#     roc_auc = dict()\n",
    "#     auc_avg = 0\n",
    "#     counting = 0\n",
    "#     for i in range(n_classes):\n",
    "#       fpr[i], tpr[i], _ = roc_curve(y_test_bin[:, i], y_score[:, i])\n",
    "#      # plt.plot(fpr[i], tpr[i], color='darkorange', lw=2)\n",
    "#       #print('AUC for Class {}: {}'.format(i+1, auc(fpr[i], tpr[i])))\n",
    "#       auc_avg += auc(fpr[i], tpr[i])\n",
    "#       counting = i+1\n",
    "#     return auc_avg/counting\n",
    "\n",
    "\n",
    "# In[3]:\n",
    "\n",
    "\n",
    "# attach the column names to the dataset\n",
    "feature=[\"duration\",\"protocol_type\",\"service\",\"flag\",\"src_bytes\",\"dst_bytes\",\"land\",\"wrong_fragment\",\"urgent\",\"hot\",\n",
    "          \"num_failed_logins\",\"logged_in\",\"num_compromised\",\"root_shell\",\"su_attempted\",\"num_root\",\"num_file_creations\",\"num_shells\",\n",
    "          \"num_access_files\",\"num_outbound_cmds\",\"is_host_login\",\"is_guest_login\",\"count\",\"srv_count\",\"serror_rate\",\"srv_serror_rate\",\n",
    "          \"rerror_rate\",\"srv_rerror_rate\",\"same_srv_rate\",\"diff_srv_rate\",\"srv_diff_host_rate\",\"dst_host_count\",\"dst_host_srv_count\", \n",
    "          \"dst_host_same_srv_rate\",\"dst_host_diff_srv_rate\",\"dst_host_same_src_port_rate\",\"dst_host_srv_diff_host_rate\",\"dst_host_serror_rate\",\n",
    "          \"dst_host_srv_serror_rate\",\"dst_host_rerror_rate\",\"dst_host_srv_rerror_rate\",\"label\",\"difficulty\"]\n",
    "# KDDTrain+_2.csv & KDDTest+_2.csv are the datafiles without the last column about the difficulty score\n",
    "# these have already been removed.\n",
    "\n",
    "train='KDDTrain+.txt'\n",
    "test='KDDTest+.txt'\n",
    "\n",
    "df=pd.read_csv(train,names=feature)\n",
    "df_test=pd.read_csv(test,names=feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimensions of the Training set: (125973, 43)\n",
      "Dimensions of the Test set: (22544, 43)\n",
      "Label distribution Training set:\n",
      "normal             67343\n",
      "neptune            41214\n",
      "satan               3633\n",
      "ipsweep             3599\n",
      "portsweep           2931\n",
      "smurf               2646\n",
      "nmap                1493\n",
      "back                 956\n",
      "teardrop             892\n",
      "warezclient          890\n",
      "pod                  201\n",
      "guess_passwd          53\n",
      "buffer_overflow       30\n",
      "warezmaster           20\n",
      "land                  18\n",
      "imap                  11\n",
      "rootkit               10\n",
      "loadmodule             9\n",
      "ftp_write              8\n",
      "multihop               7\n",
      "phf                    4\n",
      "perl                   3\n",
      "spy                    2\n",
      "Name: label, dtype: int64\n",
      "\n",
      "Label distribution Test set:\n",
      "normal             9711\n",
      "neptune            4657\n",
      "guess_passwd       1231\n",
      "mscan               996\n",
      "warezmaster         944\n",
      "apache2             737\n",
      "satan               735\n",
      "processtable        685\n",
      "smurf               665\n",
      "back                359\n",
      "snmpguess           331\n",
      "saint               319\n",
      "mailbomb            293\n",
      "snmpgetattack       178\n",
      "portsweep           157\n",
      "ipsweep             141\n",
      "httptunnel          133\n",
      "nmap                 73\n",
      "pod                  41\n",
      "buffer_overflow      20\n",
      "multihop             18\n",
      "named                17\n",
      "ps                   15\n",
      "sendmail             14\n",
      "xterm                13\n",
      "rootkit              13\n",
      "teardrop             12\n",
      "xlock                 9\n",
      "land                  7\n",
      "xsnoop                4\n",
      "ftp_write             3\n",
      "perl                  2\n",
      "udpstorm              2\n",
      "worm                  2\n",
      "phf                   2\n",
      "sqlattack             2\n",
      "loadmodule            2\n",
      "imap                  1\n",
      "Name: label, dtype: int64\n",
      "Training set:\n",
      "Feature 'protocol_type' has 3 categories\n",
      "Feature 'service' has 70 categories\n",
      "Feature 'flag' has 11 categories\n",
      "Feature 'label' has 23 categories\n",
      "\n",
      "Distribution of categories in service:\n",
      "http        40338\n",
      "private     21853\n",
      "domain_u     9043\n",
      "smtp         7313\n",
      "ftp_data     6860\n",
      "Name: service, dtype: int64\n",
      "Test set:\n",
      "Feature 'protocol_type' has 3 categories\n",
      "Feature 'service' has 64 categories\n",
      "Feature 'flag' has 11 categories\n",
      "Feature 'label' has 38 categories\n",
      "['Protocol_type_icmp', 'Protocol_type_tcp', 'Protocol_type_udp', 'service_IRC', 'service_X11', 'service_Z39_50', 'service_aol', 'service_auth', 'service_bgp', 'service_courier', 'service_csnet_ns', 'service_ctf', 'service_daytime', 'service_discard', 'service_domain', 'service_domain_u', 'service_echo', 'service_eco_i', 'service_ecr_i', 'service_efs', 'service_exec', 'service_finger', 'service_ftp', 'service_ftp_data', 'service_gopher', 'service_harvest', 'service_hostnames', 'service_http', 'service_http_2784', 'service_http_443', 'service_http_8001', 'service_imap4', 'service_iso_tsap', 'service_klogin', 'service_kshell', 'service_ldap', 'service_link', 'service_login', 'service_mtp', 'service_name', 'service_netbios_dgm', 'service_netbios_ns', 'service_netbios_ssn', 'service_netstat', 'service_nnsp', 'service_nntp', 'service_ntp_u', 'service_other', 'service_pm_dump', 'service_pop_2', 'service_pop_3', 'service_printer', 'service_private', 'service_red_i', 'service_remote_job', 'service_rje', 'service_shell', 'service_smtp', 'service_sql_net', 'service_ssh', 'service_sunrpc', 'service_supdup', 'service_systat', 'service_telnet', 'service_tftp_u', 'service_tim_i', 'service_time', 'service_urh_i', 'service_urp_i', 'service_uucp', 'service_uucp_path', 'service_vmnet', 'service_whois', 'flag_OTH', 'flag_REJ', 'flag_RSTO', 'flag_RSTOS0', 'flag_RSTR', 'flag_S0', 'flag_S1', 'flag_S2', 'flag_S3', 'flag_SF', 'flag_SH']\n",
      "   protocol_type  service  flag\n",
      "0              1       20     9\n",
      "1              2       44     9\n",
      "2              1       49     5\n",
      "3              1       24     9\n",
      "4              1       24     9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(125973, 123)\n",
      "(22544, 123)\n",
      "0    0\n",
      "1    0\n",
      "2    1\n",
      "3    0\n",
      "4    0\n",
      "Name: label, dtype: int64\n",
      "X_train has shape: (125973, 122) \n",
      "y_train has shape: (125973, 1)\n",
      "X_test has shape: (22544, 122) \n",
      "y_test has shape: (22544, 1)\n",
      "Counter({0: 67343, 1: 45927, 2: 11656, 3: 995, 4: 52})\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# shape, this gives the dimensions of the dataset\n",
    "print('Dimensions of the Training set:',df.shape)\n",
    "print('Dimensions of the Test set:',df_test.shape)\n",
    "\n",
    "\n",
    "df.drop(['difficulty'],axis=1,inplace=True)\n",
    "df_test.drop(['difficulty'],axis=1,inplace=True)\n",
    "\n",
    "\n",
    "\n",
    "print('Label distribution Training set:')\n",
    "print(df['label'].value_counts())\n",
    "print()\n",
    "print('Label distribution Test set:')\n",
    "print(df_test['label'].value_counts())\n",
    "\n",
    "\n",
    "\n",
    "# colums that are categorical and not binary yet: protocol_type (column 2), service (column 3), flag (column 4).\n",
    "# explore categorical features\n",
    "print('Training set:')\n",
    "for col_name in df.columns:\n",
    "    if df[col_name].dtypes == 'object' :\n",
    "        unique_cat = len(df[col_name].unique())\n",
    "        print(\"Feature '{col_name}' has {unique_cat} categories\".format(col_name=col_name, unique_cat=unique_cat))\n",
    "\n",
    "#see how distributed the feature service is, it is evenly distributed and therefore we need to make dummies for all.\n",
    "print()\n",
    "print('Distribution of categories in service:')\n",
    "print(df['service'].value_counts().sort_values(ascending=False).head())\n",
    "\n",
    "\n",
    "\n",
    "# Test set\n",
    "print('Test set:')\n",
    "for col_name in df_test.columns:\n",
    "    if df_test[col_name].dtypes == 'object' :\n",
    "        unique_cat = len(df_test[col_name].unique())\n",
    "        print(\"Feature '{col_name}' has {unique_cat} categories\".format(col_name=col_name, unique_cat=unique_cat))\n",
    "\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder,OneHotEncoder\n",
    "categorical_columns=['protocol_type', 'service', 'flag']\n",
    "# insert code to get a list of categorical columns into a variable, categorical_columns\n",
    "categorical_columns=['protocol_type', 'service', 'flag'] \n",
    " # Get the categorical values into a 2D numpy array\n",
    "df_categorical_values = df[categorical_columns]\n",
    "testdf_categorical_values = df_test[categorical_columns]\n",
    "df_categorical_values.head()\n",
    "\n",
    "\n",
    "# protocol type\n",
    "unique_protocol=sorted(df.protocol_type.unique())\n",
    "string1 = 'Protocol_type_'\n",
    "unique_protocol2=[string1 + x for x in unique_protocol]\n",
    "# service\n",
    "unique_service=sorted(df.service.unique())\n",
    "string2 = 'service_'\n",
    "unique_service2=[string2 + x for x in unique_service]\n",
    "# flag\n",
    "unique_flag=sorted(df.flag.unique())\n",
    "string3 = 'flag_'\n",
    "unique_flag2=[string3 + x for x in unique_flag]\n",
    "# put together\n",
    "dumcols=unique_protocol2 + unique_service2 + unique_flag2\n",
    "print(dumcols)\n",
    "\n",
    "#do same for test set\n",
    "unique_service_test=sorted(df_test.service.unique())\n",
    "unique_service2_test=[string2 + x for x in unique_service_test]\n",
    "testdumcols=unique_protocol2 + unique_service2_test + unique_flag2\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "df_categorical_values_enc=df_categorical_values.apply(LabelEncoder().fit_transform)\n",
    "print(df_categorical_values_enc.head())\n",
    "# test set\n",
    "testdf_categorical_values_enc=testdf_categorical_values.apply(LabelEncoder().fit_transform)\n",
    "\n",
    "\n",
    "\n",
    "enc = OneHotEncoder()\n",
    "df_categorical_values_encenc = enc.fit_transform(df_categorical_values_enc)\n",
    "df_cat_data = pd.DataFrame(df_categorical_values_encenc.toarray(),columns=dumcols)\n",
    "# test set\n",
    "testdf_categorical_values_encenc = enc.fit_transform(testdf_categorical_values_enc)\n",
    "testdf_cat_data = pd.DataFrame(testdf_categorical_values_encenc.toarray(),columns=testdumcols)\n",
    "\n",
    "df_cat_data.head()\n",
    "\n",
    "\n",
    "trainservice=df['service'].tolist()\n",
    "testservice= df_test['service'].tolist()\n",
    "difference=list(set(trainservice) - set(testservice))\n",
    "string = 'service_'\n",
    "difference=[string + x for x in difference]\n",
    "difference\n",
    "\n",
    "for col in difference:\n",
    "    testdf_cat_data[col] = 0\n",
    "\n",
    "testdf_cat_data.shape\n",
    "\n",
    "newdf=df.join(df_cat_data)\n",
    "newdf.drop('flag', axis=1, inplace=True)\n",
    "newdf.drop('protocol_type', axis=1, inplace=True)\n",
    "newdf.drop('service', axis=1, inplace=True)\n",
    "# test data\n",
    "newdf_test=df_test.join(testdf_cat_data)\n",
    "newdf_test.drop('flag', axis=1, inplace=True)\n",
    "newdf_test.drop('protocol_type', axis=1, inplace=True)\n",
    "newdf_test.drop('service', axis=1, inplace=True)\n",
    "print(newdf.shape)\n",
    "print(newdf_test.shape)\n",
    "\n",
    "\n",
    "# take label column\n",
    "labeldf=newdf['label']\n",
    "labeldf_test=newdf_test['label']\n",
    "# change the label column\n",
    "newlabeldf=labeldf.replace({ 'normal' : 0, 'neptune' : 1 ,'back': 1, 'land': 1, 'pod': 1, 'smurf': 1, 'teardrop': 1,'mailbomb': 1, 'apache2': 1, 'processtable': 1, 'udpstorm': 1, 'worm': 1,\n",
    "                           'ipsweep' : 2,'nmap' : 2,'portsweep' : 2,'satan' : 2,'mscan' : 2,'saint' : 2\n",
    "                           ,'ftp_write': 3,'guess_passwd': 3,'imap': 3,'multihop': 3,'phf': 3,'spy': 3,'warezclient': 3,'warezmaster': 3,'sendmail': 3,'named': 3,'snmpgetattack': 3,'snmpguess': 3,'xlock': 3,'xsnoop': 3,'httptunnel': 3,\n",
    "                           'buffer_overflow': 4,'loadmodule': 4,'perl': 4,'rootkit': 4,'ps': 4,'sqlattack': 4,'xterm': 4})\n",
    "newlabeldf_test=labeldf_test.replace({ 'normal' : 0, 'neptune' : 1 ,'back': 1, 'land': 1, 'pod': 1, 'smurf': 1, 'teardrop': 1,'mailbomb': 1, 'apache2': 1, 'processtable': 1, 'udpstorm': 1, 'worm': 1,\n",
    "                           'ipsweep' : 2,'nmap' : 2,'portsweep' : 2,'satan' : 2,'mscan' : 2,'saint' : 2\n",
    "                           ,'ftp_write': 3,'guess_passwd': 3,'imap': 3,'multihop': 3,'phf': 3,'spy': 3,'warezclient': 3,'warezmaster': 3,'sendmail': 3,'named': 3,'snmpgetattack': 3,'snmpguess': 3,'xlock': 3,'xsnoop': 3,'httptunnel': 3,\n",
    "                           'buffer_overflow': 4,'loadmodule': 4,'perl': 4,'rootkit': 4,'ps': 4,'sqlattack': 4,'xterm': 4})\n",
    "# put the new label column back\n",
    "newdf['label'] = newlabeldf\n",
    "newdf_test['label'] = newlabeldf_test\n",
    "print(newdf['label'].head())\n",
    "\n",
    "\n",
    "# Specify your selected features. Note that you'll need to modify this list according to your final processed dataframe\n",
    "#Uncomment the below lines to use these top 20 features from shap analysis\n",
    "#selected_features = [\"root_shell\",\"service_telnet\",\"num_shells\",\"service_uucp\",\"dst_host_same_src_port_rate\"\n",
    "#                     ,\"dst_host_rerror_rate\",\"dst_host_srv_serror_rate\",\"dst_host_srv_count\",\"service_private\",\"logged_in\",\n",
    "#                    \"dst_host_serror_rate\",\"serror_rate\",\"srv_serror_rate\",\"flag_S0\",\"diff_srv_rate\",\"dst_host_srv_diff_host_rate\",\"num_file_creations\",\"flag_RSTR\"#,\"dst_host_same_srv_rate\",\"service_Idap\",\"label\"]\n",
    "                     \n",
    "\n",
    "# Select those features from your dataframe\n",
    "#newdf = newdf[selected_features]\n",
    "#newdf_test = newdf_test[selected_features]\n",
    "\n",
    "# Now your dataframe only contains your selected features.\n",
    "\n",
    "# creating a dataframe with multi-class labels (Dos,Probe,R2L,U2R,normal)\n",
    "multi_data = newdf.copy()\n",
    "multi_label = pd.DataFrame(multi_data.label)\n",
    "\n",
    "multi_data_test=newdf_test.copy()\n",
    "multi_label_test = pd.DataFrame(multi_data_test.label)\n",
    "\n",
    "\n",
    "# using standard scaler for normalizing\n",
    "std_scaler = StandardScaler()\n",
    "def standardization(df,col):\n",
    "    for i in col:\n",
    "        arr = df[i]\n",
    "        arr = np.array(arr)\n",
    "        df[i] = std_scaler.fit_transform(arr.reshape(len(arr),1))\n",
    "    return df\n",
    "\n",
    "numeric_col = multi_data.select_dtypes(include='number').columns\n",
    "data = standardization(multi_data,numeric_col)\n",
    "numeric_col_test = multi_data_test.select_dtypes(include='number').columns\n",
    "data_test = standardization(multi_data_test,numeric_col_test)\n",
    "\n",
    "# label encoding (0,1,2,3,4) multi-class labels (Dos,normal,Probe,R2L,U2R)\n",
    "le2 = preprocessing.LabelEncoder()\n",
    "le2_test = preprocessing.LabelEncoder()\n",
    "enc_label = multi_label.apply(le2.fit_transform)\n",
    "enc_label_test = multi_label_test.apply(le2_test.fit_transform)\n",
    "multi_data = multi_data.copy()\n",
    "multi_data_test = multi_data_test.copy()\n",
    "\n",
    "multi_data['intrusion'] = enc_label\n",
    "multi_data_test['intrusion'] = enc_label_test\n",
    "\n",
    "#y_mul = multi_data['intrusion']\n",
    "multi_data\n",
    "multi_data_test\n",
    "\n",
    "\n",
    "\n",
    "multi_data.drop(labels= [ 'label'], axis=1, inplace=True)\n",
    "multi_data\n",
    "multi_data_test.drop(labels= [ 'label'], axis=1, inplace=True)\n",
    "multi_data_test\n",
    "\n",
    "\n",
    "y_train_multi= multi_data[['intrusion']]\n",
    "X_train_multi= multi_data.drop(labels=['intrusion'], axis=1)\n",
    "\n",
    "print('X_train has shape:',X_train_multi.shape,'\\ny_train has shape:',y_train_multi.shape)\n",
    "\n",
    "y_test_multi= multi_data_test[['intrusion']]\n",
    "X_test_multi= multi_data_test.drop(labels=['intrusion'], axis=1)\n",
    "\n",
    "print('X_test has shape:',X_test_multi.shape,'\\ny_test has shape:',y_test_multi.shape)\n",
    "\n",
    "\n",
    "from collections import Counter\n",
    "\n",
    "label_counts = Counter(y_train_multi['intrusion'])\n",
    "print(label_counts)\n",
    "\n",
    "\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "\n",
    "y_train_multi = LabelBinarizer().fit_transform(y_train_multi)\n",
    "\n",
    "y_test_multi = LabelBinarizer().fit_transform(y_test_multi)\n",
    "\n",
    "\n",
    "Y_train=y_train_multi.copy()\n",
    "X_train=X_train_multi.copy()\n",
    "\n",
    "Y_test=y_test_multi.copy()\n",
    "X_test=X_test_multi.copy()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 0 0 0]\n",
      " [0 1 0 0 0]\n",
      " [1 0 0 0 0]\n",
      " ...\n",
      " [0 1 0 0 0]\n",
      " [1 0 0 0 0]\n",
      " [0 0 1 0 0]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[-1.10249223e-01, -7.67859947e-03, -4.91864438e-03, ...,\n",
       "        -1.97262160e-02,  8.25150071e-01, -4.64315895e-02],\n",
       "       [-1.10249223e-01, -7.73736981e-03, -4.91864438e-03, ...,\n",
       "        -1.97262160e-02,  8.25150071e-01, -4.64315895e-02],\n",
       "       [-1.10249223e-01, -7.76224074e-03, -4.91864438e-03, ...,\n",
       "        -1.97262160e-02, -1.21190076e+00, -4.64315895e-02],\n",
       "       ...,\n",
       "       [-9.29714678e-02, -7.36430591e-03, -3.87394518e-03, ...,\n",
       "        -1.97262160e-02,  8.25150071e-01, -4.64315895e-02],\n",
       "       [-8.68282658e-02, -7.36430591e-03, -3.87568593e-03, ...,\n",
       "        -1.97262160e-02,  8.25150071e-01, -4.64315895e-02],\n",
       "       [ 1.61587463e-01, -7.46804833e-03,  1.06953862e-03, ...,\n",
       "        -1.97262160e-02,  8.25150071e-01, -4.64315895e-02]])"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from sklearn.datasets import make_classification\n",
    "\n",
    "# Assuming you have features X and labels Y\n",
    "# X, Y = make_classification()\n",
    "\n",
    "ros = RandomOverSampler(sampling_strategy='minority', random_state=100)\n",
    "\n",
    "X_train, Y_train = ros.fit_resample(X_train, Y_train)\n",
    "\n",
    "\n",
    "# In[33]:\n",
    "\n",
    "\n",
    "print(Y_test)\n",
    "\n",
    "\n",
    "# In[34]:\n",
    "\n",
    "\n",
    "X_train.values\n",
    "\n",
    "\n",
    "# In[35]:\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "single_class_train = np.argmax(y_train_multi, axis=1)\n",
    "single_class_test = np.argmax(y_test_multi, axis=1)\n",
    "\n",
    "\n",
    "df1 = X_train_multi.assign(Label = single_class_train)\n",
    "df2 =  X_test_multi.assign(Label = single_class_test)\n",
    "\n",
    "frames = [df1,  df2]\n",
    "\n",
    "df = pd.concat(frames,ignore_index=True)\n",
    "\n",
    "\n",
    "if feature_selection_bit == 1:\n",
    "    feature_selection = [\n",
    "                        'dst_host_same_srv_rate',\n",
    "                        'dst_host_srv_count',\n",
    "                        'dst_host_same_src_port_rate',\n",
    "                        'logged_in',\n",
    "                        'dst_host_serror_rate',\n",
    "                        'count',\n",
    "                        'srv_count',\n",
    "                        'dst_host_rerror_rate',\n",
    "                        'Label'\n",
    "                        ]\n",
    "\n",
    "    df_og = df\n",
    "    df = df[feature_selection]\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "y = df.pop('Label')\n",
    "X = df\n",
    "\n",
    "y1, y2 = pd.factorize(y)\n",
    "\n",
    "y_0 = pd.DataFrame(y1)\n",
    "y_1 = pd.DataFrame(y1)\n",
    "y_2 = pd.DataFrame(y1)\n",
    "y_3 = pd.DataFrame(y1)\n",
    "y_4 = pd.DataFrame(y1)\n",
    "\n",
    "\n",
    "# y_0 = y_0.replace(0, 0)\n",
    "# y_0 = y_0.replace(1, 1)\n",
    "y_0 = y_0.replace(2, 1)\n",
    "y_0 = y_0.replace(3, 1)\n",
    "y_0 = y_0.replace(4, 1)\n",
    "\n",
    "\n",
    "y_1 = y_1.replace(1, 999)\n",
    "y_1 = y_1.replace(0, 1)\n",
    "# y_1 = y_1.replace(1, 0)\n",
    "y_1 = y_1.replace(2, 1)\n",
    "y_1 = y_1.replace(3, 1)\n",
    "y_1 = y_1.replace(4, 1)\n",
    "y_1 = y_1.replace(999, 1)\n",
    "\n",
    "\n",
    "y_2 = y_2.replace(0, 1)\n",
    "y_2 = y_2.replace(1, 1)\n",
    "y_2 = y_2.replace(2, 0)\n",
    "y_2 = y_2.replace(3, 1)\n",
    "y_2 = y_2.replace(4, 1)\n",
    "\n",
    "\n",
    "y_3 = y_3.replace(0, 1)\n",
    "# y_3 = y_3.replace(1, 1)\n",
    "y_3 = y_3.replace(2, 1)\n",
    "y_3 = y_3.replace(3, 0)\n",
    "y_3 = y_3.replace(4, 1)\n",
    "\n",
    "\n",
    "y_4 = y_4.replace(0, 1)\n",
    "# y_4 = y_4.replace(1, 1)\n",
    "y_4 = y_4.replace(2, 1)\n",
    "y_4 = y_4.replace(3, 1)\n",
    "y_4 = y_4.replace(4, 0)\n",
    "\n",
    "\n",
    "\n",
    "df = df.assign(Label = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Divide the dataset between level 00 and level 01\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "split = 0.5 # 0.7\n",
    "\n",
    "# X_00,X_01, y_00, y_01 = sklearn.model_selection.train_test_split(X, y, train_size=split)\n",
    "X_train,X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({0: 77054, 1: 53387, 2: 14077, 3: 3880, 4: 119})\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "label_counts2 = Counter(y)\n",
    "print(label_counts2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Base learner Split\n",
    "# split = 0.7\n",
    "\n",
    "# X_train,X_test, y_train, y_test = sklearn.model_selection.train_test_split(X_00, y_00, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>count</th>\n",
       "      <th>srv_count</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16853</th>\n",
       "      <td>-0.938287</td>\n",
       "      <td>-0.900189</td>\n",
       "      <td>-0.447834</td>\n",
       "      <td>1.235694</td>\n",
       "      <td>-0.639532</td>\n",
       "      <td>-0.717045</td>\n",
       "      <td>-0.368110</td>\n",
       "      <td>-0.159292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119317</th>\n",
       "      <td>1.066401</td>\n",
       "      <td>1.258754</td>\n",
       "      <td>-0.383108</td>\n",
       "      <td>1.235694</td>\n",
       "      <td>-0.639532</td>\n",
       "      <td>-0.708312</td>\n",
       "      <td>-0.340575</td>\n",
       "      <td>-0.387635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34846</th>\n",
       "      <td>1.066401</td>\n",
       "      <td>1.258754</td>\n",
       "      <td>-0.480197</td>\n",
       "      <td>-0.809262</td>\n",
       "      <td>-0.639532</td>\n",
       "      <td>-0.708312</td>\n",
       "      <td>-0.065228</td>\n",
       "      <td>-0.387635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85041</th>\n",
       "      <td>1.066401</td>\n",
       "      <td>1.258754</td>\n",
       "      <td>-0.447834</td>\n",
       "      <td>1.235694</td>\n",
       "      <td>-0.639532</td>\n",
       "      <td>-0.699579</td>\n",
       "      <td>-0.257971</td>\n",
       "      <td>-0.387635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147372</th>\n",
       "      <td>0.668563</td>\n",
       "      <td>0.673184</td>\n",
       "      <td>2.506809</td>\n",
       "      <td>-0.890373</td>\n",
       "      <td>-0.358118</td>\n",
       "      <td>-0.070240</td>\n",
       "      <td>2.165668</td>\n",
       "      <td>-0.473593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104331</th>\n",
       "      <td>-1.071933</td>\n",
       "      <td>-0.963422</td>\n",
       "      <td>-0.480197</td>\n",
       "      <td>-0.809262</td>\n",
       "      <td>1.608759</td>\n",
       "      <td>1.501137</td>\n",
       "      <td>-0.354343</td>\n",
       "      <td>-0.387635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52755</th>\n",
       "      <td>1.066401</td>\n",
       "      <td>1.258754</td>\n",
       "      <td>-0.447834</td>\n",
       "      <td>1.235694</td>\n",
       "      <td>-0.639532</td>\n",
       "      <td>-0.682113</td>\n",
       "      <td>-0.299273</td>\n",
       "      <td>-0.387635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98535</th>\n",
       "      <td>-1.005110</td>\n",
       "      <td>-0.873089</td>\n",
       "      <td>-0.480197</td>\n",
       "      <td>-0.809262</td>\n",
       "      <td>1.608759</td>\n",
       "      <td>1.108152</td>\n",
       "      <td>-0.120298</td>\n",
       "      <td>-0.387635</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86200</th>\n",
       "      <td>-0.091863</td>\n",
       "      <td>-0.520793</td>\n",
       "      <td>-0.447834</td>\n",
       "      <td>1.235694</td>\n",
       "      <td>-0.639532</td>\n",
       "      <td>-0.725778</td>\n",
       "      <td>-0.368110</td>\n",
       "      <td>-0.355014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14986</th>\n",
       "      <td>1.066401</td>\n",
       "      <td>1.258754</td>\n",
       "      <td>-0.383108</td>\n",
       "      <td>1.235694</td>\n",
       "      <td>-0.639532</td>\n",
       "      <td>-0.437589</td>\n",
       "      <td>0.099980</td>\n",
       "      <td>-0.387635</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74258 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        dst_host_same_srv_rate  dst_host_srv_count  \\\n",
       "16853                -0.938287           -0.900189   \n",
       "119317                1.066401            1.258754   \n",
       "34846                 1.066401            1.258754   \n",
       "85041                 1.066401            1.258754   \n",
       "147372                0.668563            0.673184   \n",
       "...                        ...                 ...   \n",
       "104331               -1.071933           -0.963422   \n",
       "52755                 1.066401            1.258754   \n",
       "98535                -1.005110           -0.873089   \n",
       "86200                -0.091863           -0.520793   \n",
       "14986                 1.066401            1.258754   \n",
       "\n",
       "        dst_host_same_src_port_rate  logged_in  dst_host_serror_rate  \\\n",
       "16853                     -0.447834   1.235694             -0.639532   \n",
       "119317                    -0.383108   1.235694             -0.639532   \n",
       "34846                     -0.480197  -0.809262             -0.639532   \n",
       "85041                     -0.447834   1.235694             -0.639532   \n",
       "147372                     2.506809  -0.890373             -0.358118   \n",
       "...                             ...        ...                   ...   \n",
       "104331                    -0.480197  -0.809262              1.608759   \n",
       "52755                     -0.447834   1.235694             -0.639532   \n",
       "98535                     -0.480197  -0.809262              1.608759   \n",
       "86200                     -0.447834   1.235694             -0.639532   \n",
       "14986                     -0.383108   1.235694             -0.639532   \n",
       "\n",
       "           count  srv_count  dst_host_rerror_rate  \n",
       "16853  -0.717045  -0.368110             -0.159292  \n",
       "119317 -0.708312  -0.340575             -0.387635  \n",
       "34846  -0.708312  -0.065228             -0.387635  \n",
       "85041  -0.699579  -0.257971             -0.387635  \n",
       "147372 -0.070240   2.165668             -0.473593  \n",
       "...          ...        ...                   ...  \n",
       "104331  1.501137  -0.354343             -0.387635  \n",
       "52755  -0.682113  -0.299273             -0.387635  \n",
       "98535   1.108152  -0.120298             -0.387635  \n",
       "86200  -0.725778  -0.368110             -0.355014  \n",
       "14986  -0.437589   0.099980             -0.387635  \n",
       "\n",
       "[74258 rows x 8 columns]"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16853     0\n",
       "119317    0\n",
       "34846     0\n",
       "85041     0\n",
       "147372    0\n",
       "         ..\n",
       "104331    1\n",
       "52755     0\n",
       "98535     1\n",
       "86200     0\n",
       "14986     0\n",
       "Name: Label, Length: 74258, dtype: int64"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LEVEL 0 - Weak models - Base Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining RF Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining ADA Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining LGBM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining KNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining SVM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining MLP Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining DNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_18 (Dense)             (None, 3)                 27        \n",
      "_________________________________________________________________\n",
      "dropout_15 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_16 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_17 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_18 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_19 (Dropout)         (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 5)                 20        \n",
      "=================================================================\n",
      "Total params: 95\n",
      "Trainable params: 95\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "with open(output_file_name, \"a\") as f: print('------------START of WEAK LEARNERS (BASE MODELS) - STACK 00 -----------------', file = f)\n",
    "\n",
    "#Defining Basemodels\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining RF Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "#------------------------------------------------------------------------------\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining ADA Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#ADA\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining LGBM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "\n",
    "\n",
    "#KNN\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining KNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf=KNeighborsClassifier(n_neighbors = 5)\n",
    "\n",
    "\n",
    "#SVM\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining SVM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "\n",
    "#MLP\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining MLP Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "import time\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "\n",
    "#DNN\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining DNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "# #Model Parameters\n",
    "# dropout_rate = 0.01\n",
    "# nodes = 70\n",
    "# out_layer = 5\n",
    "# optimizer='adam'\n",
    "# loss='sparse_categorical_crossentropy'\n",
    "# epochs=1\n",
    "# batch_size=2*256\n",
    "\n",
    "#Model Parameters\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "\n",
    "\n",
    "num_columns = X_train.shape[1]\n",
    "\n",
    "dnn = tf.keras.Sequential()\n",
    "\n",
    "# Input layer\n",
    "dnn.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "# Dense layers with dropout\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# Output layer\n",
    "dnn.add(tf.keras.layers.Dense(out_layer))\n",
    "\n",
    "\n",
    "\n",
    "dnn.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "dnn.summary()\n",
    "\n",
    "\n",
    "\n",
    "# dnn = Sequential()\n",
    "# dnn.add(Dense(128, input_dim=X_train.shape[1], activation='relu'))  # Input layer\n",
    "# dnn.add(Dense(64, activation='relu'))  # Hidden layer\n",
    "# dnn.add(Dense(5))  # Output layer\n",
    "\n",
    "# dnn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "# # summary of model layers\n",
    "# dnn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #SVM\n",
    "# # Wrap SGDClassifier with MultiOutputClassifier\n",
    "# multi_target_clf = MultiOutputClassifier(clf)\n",
    "\n",
    "# # Fit the model on the training data\n",
    "# multi_target_clf.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test data\n",
    "# y_pred = clf.predict(X_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Training Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training ADA\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation scores: [0.79221654 0.7760571  0.76528414 0.76506633 0.81981011]\n",
      "Mean accuracy: 0.7836868418588648\n",
      "---------------------------------------------------------------------------------\n",
      "Training RF\n",
      "---------------------------------------------------------------------------------\n",
      "Cross-validation scores: [0.92169405 0.92162672 0.92721519 0.91805266 0.91993805]\n",
      "Mean accuracy: 0.921705332490004\n",
      "---------------------------------------------------------------------------------\n",
      "Training SVM\n",
      "---------------------------------------------------------------------------------\n",
      "Cross-validation scores: [0.86244277 0.86702128 0.86419337 0.86957107 0.86559828]\n",
      "Mean accuracy: 0.8657653537483014\n",
      "---------------------------------------------------------------------------------\n",
      "Training KNN\n",
      "---------------------------------------------------------------------------------\n",
      "Cross-validation scores: [0.97273095 0.97326959 0.97300027 0.97192108 0.97320046]\n",
      "Mean accuracy: 0.9728244697217969\n",
      "---------------------------------------------------------------------------------\n",
      "Training LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "Cross-validation scores: [0.97993536 0.98027202 0.97946405 0.97663457 0.98027069]\n",
      "Mean accuracy: 0.9793153367258134\n",
      "---------------------------------------------------------------------------------\n",
      "Training MLP\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation scores: [0.96619984 0.96586318 0.96559386 0.96592822 0.96525486]\n",
      "Mean accuracy: 0.9657679933084475\n",
      "---------------------------------------------------------------------------------\n",
      "Training DNN\n",
      "---------------------------------------------------------------------------------\n",
      "Epoch 1/100\n",
      "465/465 [==============================] - 1s 1ms/step - loss: 4.1218 - accuracy: 0.4279 - val_loss: 1.2861 - val_accuracy: 0.6273\n",
      "Epoch 2/100\n",
      "465/465 [==============================] - 1s 1ms/step - loss: 2.1057 - accuracy: 0.4985 - val_loss: 1.3036 - val_accuracy: 0.5171\n",
      "Epoch 3/100\n",
      "465/465 [==============================] - 1s 1ms/step - loss: 1.6255 - accuracy: 0.5032 - val_loss: 1.2711 - val_accuracy: 0.5150\n",
      "Epoch 4/100\n",
      "465/465 [==============================] - 1s 1ms/step - loss: 1.4785 - accuracy: 0.5070 - val_loss: 1.2524 - val_accuracy: 0.5150\n",
      "Epoch 5/100\n",
      "465/465 [==============================] - 1s 1ms/step - loss: 1.4317 - accuracy: 0.5135 - val_loss: 1.2452 - val_accuracy: 0.5151\n",
      "Epoch 6/100\n",
      "465/465 [==============================] - 1s 1ms/step - loss: 1.3869 - accuracy: 0.5159 - val_loss: 1.0902 - val_accuracy: 0.5781\n",
      "Epoch 7/100\n",
      "465/465 [==============================] - 1s 1ms/step - loss: 1.3633 - accuracy: 0.5072 - val_loss: 1.1709 - val_accuracy: 0.5518\n",
      "Epoch 8/100\n",
      "465/465 [==============================] - 1s 1ms/step - loss: 1.3339 - accuracy: 0.5091 - val_loss: 1.1381 - val_accuracy: 0.5178\n",
      "Epoch 9/100\n",
      "465/465 [==============================] - 1s 1ms/step - loss: 1.2929 - accuracy: 0.5097 - val_loss: 1.2676 - val_accuracy: 0.5246\n",
      "Epoch 10/100\n",
      "465/465 [==============================] - 1s 1ms/step - loss: 1.3117 - accuracy: 0.5085 - val_loss: 1.1368 - val_accuracy: 0.5204\n",
      "Epoch 11/100\n",
      "465/465 [==============================] - 1s 1ms/step - loss: 1.3094 - accuracy: 0.5146 - val_loss: 1.1274 - val_accuracy: 0.5246\n"
     ]
    }
   ],
   "source": [
    "#Training Basemodels\n",
    "import joblib\n",
    "from sklearn.model_selection import StratifiedKFold, cross_val_score\n",
    "n_splits = 5  # You can adjust the number of folds as needed\n",
    "\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Training Model')\n",
    "with open(output_file_name, \"a\") as f: print('Training weak models - level 0', file = f)\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_ada == 1 and load_model_ada == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "\n",
    "\n",
    "    start = time.time()\n",
    "    ada = abc.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # Create the StratifiedKFold object\n",
    "    stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # Perform cross-validation\n",
    "    cv_scores = cross_val_score(ada, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # Print the cross-validation scores\n",
    "    print(\"Cross-validation scores:\", cv_scores)\n",
    "    print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "    # Assuming 'model' is your trained model\n",
    "    joblib.dump(ada, 'ada_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_rf == 1 and load_model_rf == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    model_rf = rf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # Create the StratifiedKFold object\n",
    "    stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # Perform cross-validation\n",
    "    cv_scores = cross_val_score(model_rf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # Print the cross-validation scores\n",
    "    print(\"Cross-validation scores:\", cv_scores)\n",
    "    print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(model_rf, 'rf_base_model.joblib')\n",
    "\n",
    "if use_model_svm == 1 and load_model_svm == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training SVM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training SVM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #SVM\n",
    "\n",
    "    start = time.time()\n",
    "    clf.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "    # clf.score(X_train, y_train)\n",
    "    time_taken = end - start\n",
    "\n",
    "    # Create the StratifiedKFold object\n",
    "    stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # Perform cross-validation\n",
    "    cv_scores = cross_val_score(clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # Print the cross-validation scores\n",
    "    print(\"Cross-validation scores:\", cv_scores)\n",
    "    print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(clf, 'svm_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_knn == 1 and load_model_knn == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    knn_clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # Create the StratifiedKFold object\n",
    "    stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # Perform cross-validation\n",
    "    cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # Print the cross-validation scores\n",
    "    print(\"Cross-validation scores:\", cv_scores)\n",
    "    print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(knn_clf, 'knn_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_lgbm == 1 and load_model_lgbm == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    lgbm.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # Create the StratifiedKFold object\n",
    "    stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # Perform cross-validation\n",
    "    cv_scores = cross_val_score(lgbm, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # Print the cross-validation scores\n",
    "    print(\"Cross-validation scores:\", cv_scores)\n",
    "    print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(lgbm, 'lgbm_base_model.joblib')\n",
    "\n",
    "if use_model_mlp == 1 and load_model_mlp == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    start = time.time()\n",
    "    MLP = mlp.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # Create the StratifiedKFold object\n",
    "    stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # Perform cross-validation\n",
    "    cv_scores = cross_val_score(MLP, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # Print the cross-validation scores\n",
    "    print(\"Cross-validation scores:\", cv_scores)\n",
    "    print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(MLP, 'mlp_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_dnn == 1 and load_model_dnn == 0:\n",
    "    from keras.callbacks import EarlyStopping\n",
    "\n",
    "    # Define EarlyStopping callback\n",
    "    early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    # Convert Y_test back to its original format\n",
    "    # y_test = np.argmax(Y_test, axis=1)\n",
    "\n",
    "    # Start the timer\n",
    "    start = time.time()\n",
    "    # dnn.fit(X_train, y_train, epochs=epochs, batch_size=batch_size)\n",
    "    dnn.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # End the timer\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(dnn, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    dnn.save(\"DNN_base_model.h5\")\n",
    "\n",
    "    # Calculate the time taken and print it out\n",
    "    # print(f'Time taken for training: {time_taken} seconds')\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from keras.models import Sequential\n",
    "# from keras.layers import Dense\n",
    "# from keras.wrappers.scikit_learn import KerasClassifier\n",
    "# from sklearn.model_selection import GridSearchCV\n",
    "# from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "# # Define your Keras model as a function\n",
    "# def create_model(optimizer='adam', hidden_layer_size=16):\n",
    "#     # model = Sequential()\n",
    "#     # model.add(Dense(hidden_layer_size, input_dim=input_size, activation='relu'))\n",
    "#     # model.add(Dense(1, activation='sigmoid'))\n",
    "#     # model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "        \n",
    "#     dnn = tf.keras.Sequential()\n",
    "\n",
    "#     # Input layer\n",
    "#     dnn.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "#     # Dense layers with dropout\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     # Output layer\n",
    "#     dnn.add(tf.keras.layers.Dense(out_layer))\n",
    "\n",
    "\n",
    "\n",
    "#     dnn.compile(optimizer=optimizer, loss=loss)\n",
    "\n",
    "#     dnn.summary()\n",
    "#     return dnn\n",
    "\n",
    "# # Create a KerasClassifier\n",
    "# dnn = KerasClassifier(build_fn=create_model, epochs=10, batch_size=32, verbose=0)\n",
    "\n",
    "# # Define the parameter grid for GridSearchCV\n",
    "# param_grid = {\n",
    "#     'optimizer': ['adam', 'sgd'],\n",
    "#     'hidden_layer_size': [8, 16, 32]\n",
    "# }\n",
    "\n",
    "# # Create the StratifiedKFold\n",
    "# cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "# # Create GridSearchCV\n",
    "# grid = GridSearchCV(estimator=dnn, param_grid=param_grid, cv=cv, scoring='accuracy')\n",
    "# grid_result = grid.fit(X_train, y_train)\n",
    "\n",
    "# # Print the best parameters and best accuracy\n",
    "# print(\"Best Parameters: \", grid_result.best_params_)\n",
    "# print(\"Best Accuracy: \", grid_result.best_score_)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stratified_kfold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading Models\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "if load_model_ada == 1:\n",
    "    ada = joblib.load('ada_base_model.joblib')\n",
    "\n",
    "if load_model_svm == 1:\n",
    "    clf =  joblib.load('svm_base_model.joblib')\n",
    "\n",
    "if load_model_dnn == 1:\n",
    "    dnn = load_model(\"DNN_base_model.h5\")\n",
    "\n",
    "if load_model_knn == 1:\n",
    "    knn_clf = joblib.load('knn_base_model.joblib')\n",
    "\n",
    "if load_model_mlp == 1:\n",
    "    MLP = joblib.load('mlp_base_model.joblib')\n",
    "\n",
    "if load_model_rf == 1:\n",
    "    rf = joblib.load('rf_base_model.joblib')\n",
    "\n",
    "if load_model_lgbm == 1:\n",
    "    lgbm = joblib.load('lgbm_base_model.joblib')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions on the test data\n",
    "# preds_svm = clf.predict(X_test)\n",
    "\n",
    "\n",
    "\n",
    "# y_scores = y_pred\n",
    "# y_true = y_test\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Base leaners predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Prediction RF\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Prediction SVM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction DNN\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction ADA\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction MLP\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction KNN\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "with open(output_file_name, \"a\") as f: print('Generating Predictions', file = f)\n",
    "\n",
    "if use_model_rf == 1:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    preds_rf = rf.predict(X_test)\n",
    "    preds_rf_prob = rf.predict_proba(X_test)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_svm == 1:\n",
    "\n",
    "    print('Prediction SVM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction SVM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #SVM\n",
    "    start = time.time()\n",
    "    preds_svm = clf.predict(X_test)\n",
    "    # preds_svm_prob = clf.predict_proba(X_test)\n",
    "\n",
    "    #Since SVM does not deal with prob by nature we use a meta learner\n",
    "    # https://stackoverflow.com/questions/55250963/how-to-get-probabilities-for-sgdclassifier-linearsvm\n",
    "\n",
    "    model = CalibratedClassifierCV(clf)\n",
    "\n",
    "    model.fit(X, y)\n",
    "    preds_svm_prob = model.predict_proba(X)\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_lgbm == 1:\n",
    "\n",
    "    print('Prediction LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #LGBM\n",
    "    start = time.time()\n",
    "    preds_lgbm = lgbm.predict(X_test)\n",
    "    preds_lgbm_prob = lgbm.predict_proba(X_test)\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_dnn == 1:\n",
    "\n",
    "    print('Prediction DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #DNN\n",
    "    start = time.time()\n",
    "    pred_dnn = dnn.predict(X_test)\n",
    "    preds_dnn_prob = pred_dnn\n",
    "    preds_dnn = np.argmax(pred_dnn,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_ada == 1:\n",
    "\n",
    "    print('Prediction ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "    start = time.time()\n",
    "    preds_ada = ada.predict(X_test)\n",
    "    preds_ada_prob = ada.predict_proba(X_test)\n",
    "\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_mlp == 1:\n",
    "\n",
    "    #MLP\n",
    "    start = time.time()\n",
    "    y_pred = MLP.predict_proba(X_test)\n",
    "    preds_mlp_prob = y_pred\n",
    "    preds_mlp = np.argmax(y_pred,axis = 1)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_knn == 1:\n",
    "\n",
    "    #KNN\n",
    "    start = time.time()\n",
    "    preds_knn =knn_clf.predict(X_test)\n",
    "    preds_knn_prob =knn_clf.predict_proba(X_test)\n",
    "\n",
    "    preds_knn\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.calibration import CalibratedClassifierCV\n",
    "# model = CalibratedClassifierCV(clf)\n",
    "\n",
    "# model.fit(X, y)\n",
    "# preds_svm_prob = model.predict_proba(X)\n",
    "\n",
    "# print(preds_ada_prob)\n",
    "# print(preds_knn_prob)\n",
    "# print(preds_dnn_prob)\n",
    "# print(preds_mlp_prob)\n",
    "# print(preds_rf_prob)\n",
    "# print(preds_svm_prob)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[7.86755451e-01 9.15808640e-02 7.78774782e-02 4.31012582e-02\n",
      "  6.84948135e-04]\n",
      " [2.62305098e-01 2.71329463e-02 6.45857927e-01 6.23936481e-02\n",
      "  2.31038051e-03]\n",
      " [1.69951428e-02 9.10446456e-01 5.10282199e-02 2.14799179e-02\n",
      "  5.02632551e-05]\n",
      " ...\n",
      " [9.42972082e-01 3.31526500e-02 1.34223706e-03 2.24056309e-02\n",
      "  1.27400343e-04]\n",
      " [9.12741440e-01 3.97164126e-02 1.80198033e-02 2.95111286e-02\n",
      "  1.12159303e-05]\n",
      " [5.57736620e-01 3.72736907e-01 5.55977328e-02 1.35376604e-02\n",
      "  3.91079556e-04]]\n",
      "[0 2 1 ... 0 0 0]\n",
      "[1 1 1 ... 0 0 1]\n"
     ]
    }
   ],
   "source": [
    "print(preds_svm_prob)\n",
    "preds_3 = np.argmax(preds_svm_prob,axis = 1)\n",
    "print(preds_3)\n",
    "\n",
    "print(preds_svm)\n",
    "# print(y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### METRICS - Base Learners"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "\n",
    "\n",
    "# >>> \n",
    "# >>> roc_auc_score(y, clf.predict_proba(X)[:, 1])\n",
    "# 0.99...\n",
    "# >>> roc_auc_score(y, clf.decision_function(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.3159299\ttest: 1.3158964\tbest: 1.3158964 (0)\ttotal: 20.6ms\tremaining: 2.04s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10:\tlearn: 0.4738781\ttest: 0.4730596\tbest: 0.4730596 (10)\ttotal: 165ms\tremaining: 1.34s\n",
      "20:\tlearn: 0.2726956\ttest: 0.2724286\tbest: 0.2724286 (20)\ttotal: 259ms\tremaining: 973ms\n",
      "30:\tlearn: 0.1958611\ttest: 0.1957983\tbest: 0.1957983 (30)\ttotal: 345ms\tremaining: 768ms\n",
      "40:\tlearn: 0.1596632\ttest: 0.1600648\tbest: 0.1600648 (40)\ttotal: 434ms\tremaining: 624ms\n",
      "50:\tlearn: 0.1379342\ttest: 0.1387839\tbest: 0.1387839 (50)\ttotal: 521ms\tremaining: 500ms\n",
      "60:\tlearn: 0.1246023\ttest: 0.1257964\tbest: 0.1257964 (60)\ttotal: 608ms\tremaining: 388ms\n",
      "70:\tlearn: 0.1152139\ttest: 0.1167232\tbest: 0.1167232 (70)\ttotal: 690ms\tremaining: 282ms\n",
      "80:\tlearn: 0.1070068\ttest: 0.1086429\tbest: 0.1086429 (80)\ttotal: 777ms\tremaining: 182ms\n",
      "90:\tlearn: 0.1009714\ttest: 0.1028407\tbest: 0.1028407 (90)\ttotal: 868ms\tremaining: 85.9ms\n",
      "99:\tlearn: 0.0961986\ttest: 0.0982917\tbest: 0.0982917 (99)\ttotal: 954ms\tremaining: 0us\n",
      "\n",
      "bestTest = 0.09829174137\n",
      "bestIteration = 99\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2       3    4\n",
      "0  37688.0    311.0   315.0   349.0  0.0\n",
      "1    316.0  26111.0   150.0    26.0  0.0\n",
      "2    285.0     51.0  6677.0    22.0  0.0\n",
      "3    316.0      9.0    26.0  1544.0  0.0\n",
      "4     32.0      2.0     0.0    29.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9698487725393555\n",
      "Precision total:  0.7353223075254655\n",
      "Recall total:  0.7440350494273591\n",
      "F1 total:  0.7396009841504471\n",
      "BACC total:  0.7440350494273591\n",
      "MCC total:  0.94903386668609\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "\n",
    "cat_00 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "# Fit the model\n",
    "cat_00.fit(X_train, y_train, eval_set=(X_test, y_test), verbose=10)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_cat = cat_00.predict(X_test)\n",
    "preds_cat_prob = cat_00.predict_proba(X_test)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Catboost base model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_cat\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "    Acc = accuracy_score(y_test, pred_label)\n",
    "    cat_acc_00 = Acc\n",
    "    Precision = precision_score(y_test, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test, pred_label)\n",
    "\n",
    "    cat_acc_00 = Acc\n",
    "    cat_pre_00 = Precision\n",
    "    cat_rec_00 = Recall\n",
    "    cat_f1_00 = F1\n",
    "    cat_bacc_00 = BACC\n",
    "    cat_mcc_00 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# from sklearn.metrics import accuracy_score\n",
    "\n",
    "# Assuming you have your features and labels as X and y\n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_00 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_xgb = xgb_00.predict(dtest)\n",
    "# preds_xgb_prob = xgb_00.predict_proba(dtest)\n",
    "\n",
    "\n",
    "# Get class probabilities\n",
    "# Assuming binary classification, get the probability for the positive class (class 1)\n",
    "preds_xgb_margin = xgb_00.predict(dtest, output_margin=True)\n",
    "preds_xgb_prob = 1 / (1 + np.exp(-preds_xgb_margin))\n",
    "\n",
    "# Print or use positive_class_probabilities as needed\n",
    "# print(positive_class_probabilities)\n",
    "\n",
    "\n",
    "# Convert predicted probabilities to class labels (if necessary)\n",
    "# y_pred_labels = [round(value) for value in y_pred]\n",
    "\n",
    "# Evaluate the accuracy\n",
    "# accuracy = accuracy_score(y_test, y_pred)\n",
    "# print(\"Accuracy: %.2f%%\" % (accuracy * 100.0))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0      1.0     2.0     3.0  4.0\n",
      "0.0  37426.0    409.0   498.0   329.0  1.0\n",
      "1.0    790.0  25636.0   130.0    47.0  0.0\n",
      "2.0    309.0    130.0  6530.0    66.0  0.0\n",
      "3.0    624.0      8.0    32.0  1231.0  0.0\n",
      "4.0     34.0      3.0     2.0    24.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy total:  0.9537295142676309\n",
      "Precision total:  0.7135013155208901\n",
      "Recall total:  0.7018953174003261\n",
      "F1 total:  0.7072374594814604\n",
      "BACC total:  0.7018953174003261\n",
      "MCC total:  0.9215446051414945\n"
     ]
    }
   ],
   "source": [
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('xgboost base model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_xgb\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "    Acc = accuracy_score(y_test, pred_label)\n",
    "    xgb_acc_00 = Acc\n",
    "    Precision = precision_score(y_test, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test, pred_label)\n",
    "    xgb_acc_00 = Acc\n",
    "    xgb_pre_00 = Precision\n",
    "    xgb_rec_00 = Recall\n",
    "    xgb_f1_00 = F1\n",
    "    xgb_bacc_00 = BACC\n",
    "    xgb_mcc_00 = MCC\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_test\n",
    "# pred_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2     3    4\n",
      "0  37798.0    292.0   570.0   3.0  0.0\n",
      "1   1859.0  24621.0   123.0   0.0  0.0\n",
      "2    725.0    548.0  5762.0   0.0  0.0\n",
      "3   1806.0      3.0    66.0  20.0  0.0\n",
      "4     59.0      0.0     4.0   0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy total:  0.9184206628152817\n",
      "Precision total:  0.7228431075942422\n",
      "Recall total:  0.5465452043853987\n",
      "F1 total:  0.5501545271989657\n",
      "BACC total:  0.5465452043853987\n",
      "MCC total:  0.8604974480981238\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "#RF\n",
    "if use_model_rf == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('RF base model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_rf\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "    Acc = accuracy_score(y_test, pred_label)\n",
    "    rf_acc_00 = Acc\n",
    "    Precision = precision_score(y_test, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test, pred_label)\n",
    "\n",
    "    rf_acc_00 = Acc\n",
    "    rf_pre_00 = Precision\n",
    "    rf_rec_00 = Recall\n",
    "    rf_f1_00 = F1\n",
    "    rf_bacc_00 = BACC\n",
    "    rf_mcc_00 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         0        1    2      3    4\n",
      "0  31753.0   6153.0  5.0  752.0  0.0\n",
      "1  11900.0  14680.0  0.0   23.0  0.0\n",
      "2   2555.0   4480.0  0.0    0.0  0.0\n",
      "3   1252.0    621.0  0.0   22.0  0.0\n",
      "4     34.0     29.0  0.0    0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.625580737688361\n",
      "Precision total:  0.25231843905556384\n",
      "Recall total:  0.27694062367480204\n",
      "F1 total:  0.2623953784372187\n",
      "BACC total:  0.27694062367480204\n",
      "MCC total:  0.3174545651795549\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "if use_model_dnn == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('DNN base model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_dnn\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "    Acc = accuracy_score(y_test, pred_label)\n",
    "    dnn_acc_00 = Acc\n",
    "\n",
    "    Precision = precision_score(y_test, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test, pred_label)\n",
    "\n",
    "    dnn_acc_00 = Acc\n",
    "    dnn_pre_00 = Precision\n",
    "    dnn_rec_00 = Recall\n",
    "    dnn_f1_00 = F1\n",
    "    dnn_bacc_00 = BACC\n",
    "    dnn_mcc_00 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3     4\n",
      "0  32437.0   4128.0  1575.0  496.0  27.0\n",
      "1   1095.0  18826.0  6484.0  198.0   0.0\n",
      "2   2199.0    289.0  4494.0   53.0   0.0\n",
      "3    835.0     70.0    26.0  952.0  12.0\n",
      "4     30.0      1.0     1.0   31.0   0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy total:  0.7636650103017816\n",
      "Precision total:  0.5202748124787877\n",
      "Recall total:  0.5375625355482514\n",
      "F1 total:  0.5199528225900233\n",
      "BACC total:  0.5375625355482514\n",
      "MCC total:  0.6197764100319277\n"
     ]
    }
   ],
   "source": [
    "#ADA\n",
    "if use_model_ada == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('ADA base model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_ada\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "    Acc = accuracy_score(y_test, pred_label)\n",
    "    ada_acc_00 = Acc\n",
    "\n",
    "    Precision = precision_score(y_test, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test, pred_label)\n",
    "\n",
    "\n",
    "    ada_acc_00 = Acc\n",
    "    ada_pre_00 = Precision\n",
    "    ada_rec_00 = Recall\n",
    "    ada_f1_00 = F1\n",
    "    ada_bacc_00 = BACC\n",
    "    ada_mcc_00 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2    3    4\n",
      "0  36476.0    650.0  1537.0  0.0  0.0\n",
      "1   2437.0  24068.0    98.0  0.0  0.0\n",
      "2   1673.0   1869.0  3493.0  0.0  0.0\n",
      "3   1616.0     82.0   194.0  3.0  0.0\n",
      "4     53.0      5.0     5.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8623466515843198\n",
      "Precision total:  0.48425063000684077\n",
      "Recall total:  0.46893233984628824\n",
      "F1 total:  0.4740355799030905\n",
      "BACC total:  0.46893233984628824\n",
      "MCC total:  0.7608278841848335\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "#SVM\n",
    "if use_model_svm == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('SVM base model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_svm\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "    Acc = accuracy_score(y_test, pred_label)\n",
    "    Precision = precision_score(y_test, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test, pred_label)\n",
    "\n",
    "    svm_acc_00 = Acc\n",
    "    svm_pre_00 = Precision\n",
    "    svm_rec_00 = Recall\n",
    "    svm_f1_00 = F1\n",
    "    svm_bacc_00 = BACC\n",
    "    svm_mcc_00 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2       3    4\n",
      "0  37695.0    326.0   303.0   338.0  1.0\n",
      "1    172.0  26359.0    37.0    35.0  0.0\n",
      "2    273.0     34.0  6693.0    35.0  0.0\n",
      "3    277.0      4.0    17.0  1597.0  0.0\n",
      "4     45.0      4.0     1.0    12.0  1.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy total:  0.9742253464226558\n",
      "Precision total:  0.8414572813380907\n",
      "Recall total:  0.7551588503706606\n",
      "F1 total:  0.7527130001463106\n",
      "BACC total:  0.7551588503706606\n",
      "MCC total:  0.9564517302067617\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "if use_model_knn == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('KNN base model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_knn\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "    Acc = accuracy_score(y_test, pred_label)\n",
    "    Precision = precision_score(y_test, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test, pred_label)\n",
    "\n",
    "\n",
    "    knn_acc_00 = Acc\n",
    "    knn_pre_00 = Precision\n",
    "    knn_rec_00 = Recall\n",
    "    knn_f1_00 = F1\n",
    "    knn_bacc_00 = BACC\n",
    "    knn_mcc_00 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2       3    4\n",
      "0  37290.0    365.0   520.0   487.0  1.0\n",
      "1    281.0  26190.0    86.0    46.0  0.0\n",
      "2    195.0     40.0  6761.0    39.0  0.0\n",
      "3    277.0     11.0    29.0  1578.0  0.0\n",
      "4     38.0      3.0     3.0    19.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy total:  0.967142029922299\n",
      "Precision total:  0.7209556541600184\n",
      "Recall total:  0.7485466016871467\n",
      "F1 total:  0.7339114879144161\n",
      "BACC total:  0.7485466016871467\n",
      "MCC total:  0.9448205272876481\n"
     ]
    }
   ],
   "source": [
    "#MLP\n",
    "if use_model_mlp == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('MLP base model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_mlp\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "    Acc = accuracy_score(y_test, pred_label)\n",
    "    Precision = precision_score(y_test, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test, pred_label)\n",
    "\n",
    "    mlp_acc_00 = Acc\n",
    "    mlp_pre_00 = Precision\n",
    "    mlp_rec_00 = Recall\n",
    "    mlp_f1_00 = F1\n",
    "    mlp_bacc_00 = BACC\n",
    "    mlp_mcc_00 = MCC\n",
    "\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2       3     4\n",
      "0  37943.0    230.0   187.0   279.0  24.0\n",
      "1    162.0  26395.0    16.0    21.0   9.0\n",
      "2    182.0     17.0  6807.0    22.0   7.0\n",
      "3    216.0      3.0    17.0  1628.0  31.0\n",
      "4     36.0      0.0     0.0    23.0   4.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy total:  0.9800428230921504\n",
      "Precision total:  0.7644634850194615\n",
      "Recall total:  0.7727488923116625\n",
      "F1 total:  0.7684486696844631\n",
      "BACC total:  0.7727488923116625\n",
      "MCC total:  0.9662701374480818\n"
     ]
    }
   ],
   "source": [
    "#lgbm\n",
    "\n",
    "if use_model_lgbm == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('LGBM base model', file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    pred_label = preds_lgbm\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    # Acc = ACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # Precision = PRECISION(TP_total, FP_total)\n",
    "    # Recall = RECALL(TP_total, FN_total)\n",
    "    # F1 = F1(Recall,Precision)\n",
    "    # BACC = BACC(TP_total,TN_total, FP_total, FN_total)\n",
    "    # MCC = MCC(TP_total,TN_total, FP_total, FN_total)\n",
    "\n",
    "    Acc = accuracy_score(y_test, pred_label)\n",
    "    Precision = precision_score(y_test, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test, pred_label)\n",
    "    lgbm_acc_00 = Acc\n",
    "    lgbm_pre_00 = Precision\n",
    "    lgbm_rec_00 = Recall\n",
    "    lgbm_f1_00 = F1\n",
    "    lgbm_bacc_00 = BACC\n",
    "    lgbm_mcc_00 = MCC\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training the stronger model - STACK level 01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "74259 74259\n"
     ]
    }
   ],
   "source": [
    "print(len(preds_dnn_prob), len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "122758    1\n",
      "121595    1\n",
      "89390     1\n",
      "9149      2\n",
      "69229     0\n",
      "         ..\n",
      "25572     1\n",
      "121751    1\n",
      "69868     0\n",
      "75733     0\n",
      "92061     1\n",
      "Name: Label, Length: 74259, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        index  Label\n",
      "0      122758      1\n",
      "1      121595      1\n",
      "2       89390      1\n",
      "3        9149      2\n",
      "4       69229      0\n",
      "...       ...    ...\n",
      "74254   25572      1\n",
      "74255  121751      1\n",
      "74256   69868      0\n",
      "74257   75733      0\n",
      "74258   92061      1\n",
      "\n",
      "[74259 rows x 2 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0        122758\n",
       "1        121595\n",
       "2         89390\n",
       "3          9149\n",
       "4         69229\n",
       "          ...  \n",
       "74254     25572\n",
       "74255    121751\n",
       "74256     69868\n",
       "74257     75733\n",
       "74258     92061\n",
       "Name: index, Length: 74259, dtype: int64"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_from_series = y_test.to_frame()\n",
    "y_test_reset_index = df_from_series.reset_index()\n",
    "# y_test2 = y_test.reset_index(inplace=True)\n",
    "print(y_test_reset_index)\n",
    "y_test_reset_index.pop('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_reset_index.values[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_dnn_2 = []\n",
    "preds_svm_2 = []\n",
    "preds_rf_2 = []\n",
    "preds_mlp_2 = []\n",
    "preds_ada_2 = []\n",
    "preds_knn_2 = []\n",
    "preds_lgbm_2 = []\n",
    "preds_cat_2 = []\n",
    "preds_xgb_2 = []\n",
    "\n",
    "for i in range(0,len(preds_dnn_prob)):  \n",
    "    # print(i)\n",
    "    # print(preds_dnn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_dnn_2.append(preds_dnn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_svm_2.append(preds_svm_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_rf_2.append(preds_rf_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_mlp_2.append(preds_mlp_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_ada_2.append(preds_ada_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_knn_2.append(preds_knn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_lgbm_2.append(preds_lgbm_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_cat_2.append(preds_cat_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_xgb_2.append(preds_xgb_prob[i][y_test_reset_index.values[i][0]])\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 1. 1. ... 1. 1. 1.]\n",
      " [0. 1. 1. ... 1. 1. 1.]\n",
      " [1. 1. 1. ... 1. 1. 1.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 1. 1. ... 1. 1. 1.]]\n",
      "[[0.22791725 0.99114636 0.99997456 ... 0.99857844 0.99629992 1.        ]\n",
      " [0.24015954 0.99114636 0.99997856 ... 0.99857844 0.99629992 1.        ]\n",
      " [0.25012285 0.99114636 0.9999713  ... 0.99819891 0.99629992 1.        ]\n",
      " ...\n",
      " [0.26432016 0.97419508 0.99980602 ... 0.99656191 0.99159497 0.        ]\n",
      " [0.34628952 0.46600298 0.89730248 ... 0.56460065 0.73466969 0.        ]\n",
      " [0.15684691 0.94299287 0.99947676 ... 0.98764666 0.9884575  1.        ]]\n"
     ]
    }
   ],
   "source": [
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('------------START of STRONGER LEARNER - STACK 01 -----------------', file = f)\n",
    "\n",
    "\n",
    "# Stack the vectors horizontally to create a matrix\n",
    "column_features = ['dnn','rf','lgbm','ada','knn','mlp','svm','cat','xgb','label']\n",
    "training_matrix2 = np.column_stack((\n",
    "                          preds_dnn_2,\n",
    "                          preds_rf_2,\n",
    "                          preds_lgbm_2,\n",
    "                          preds_ada_2,\n",
    "                          preds_knn_2, \n",
    "                          preds_mlp_2,\n",
    "                          preds_svm_2,\n",
    "                          preds_cat_2,\n",
    "                          preds_xgb_2,\n",
    "                          y_test\n",
    "                          ))\n",
    "\n",
    "training_matrix = np.column_stack((\n",
    "                          preds_dnn,\n",
    "                          preds_rf,\n",
    "                          preds_lgbm,\n",
    "                          preds_ada,\n",
    "                          preds_knn, \n",
    "                          preds_mlp,\n",
    "                          preds_svm,\n",
    "                          preds_cat,\n",
    "                          preds_xgb,\n",
    "                        #   preds\n",
    "                          y_test\n",
    "                          ))\n",
    "# Print the resulting matrix\n",
    "print(training_matrix)\n",
    "print(training_matrix2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_level_00_0 = pd.DataFrame(training_matrix, columns=column_features)\n",
    "df_level_00_1 = pd.DataFrame(training_matrix2, columns=column_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Assuming df is your DataFrame\n",
    "if feature_selection_bit == 1:\n",
    "\n",
    "    df_level_00_1.to_csv('base_models_prob_feature_selection.csv', index=False)\n",
    "    df_level_00_0.to_csv('base_models_class_feature_selection.csv', index=False)\n",
    "    \n",
    "if feature_selection_bit == 0:\n",
    "\n",
    "    df_level_00_1.to_csv('base_models_prob_all_features.csv', index=False)\n",
    "    df_level_00_0.to_csv('base_models_class_all_features.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.227917</td>\n",
       "      <td>0.991146</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.288546</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.091581</td>\n",
       "      <td>0.998578</td>\n",
       "      <td>0.996300</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.240160</td>\n",
       "      <td>0.991146</td>\n",
       "      <td>0.999979</td>\n",
       "      <td>0.288546</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.027133</td>\n",
       "      <td>0.998578</td>\n",
       "      <td>0.996300</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.250123</td>\n",
       "      <td>0.991146</td>\n",
       "      <td>0.999971</td>\n",
       "      <td>0.278205</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.910446</td>\n",
       "      <td>0.998199</td>\n",
       "      <td>0.996300</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.116905</td>\n",
       "      <td>0.563712</td>\n",
       "      <td>0.994721</td>\n",
       "      <td>0.233897</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.979222</td>\n",
       "      <td>0.001156</td>\n",
       "      <td>0.960457</td>\n",
       "      <td>0.951862</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.108086</td>\n",
       "      <td>0.828218</td>\n",
       "      <td>0.999755</td>\n",
       "      <td>0.295850</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999907</td>\n",
       "      <td>0.967048</td>\n",
       "      <td>0.997023</td>\n",
       "      <td>0.977432</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74254</th>\n",
       "      <td>0.239538</td>\n",
       "      <td>0.991146</td>\n",
       "      <td>0.999975</td>\n",
       "      <td>0.288546</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.014842</td>\n",
       "      <td>0.998425</td>\n",
       "      <td>0.996300</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74255</th>\n",
       "      <td>0.161583</td>\n",
       "      <td>0.958180</td>\n",
       "      <td>0.999924</td>\n",
       "      <td>0.297158</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.912447</td>\n",
       "      <td>0.997329</td>\n",
       "      <td>0.989533</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74256</th>\n",
       "      <td>0.264320</td>\n",
       "      <td>0.974195</td>\n",
       "      <td>0.999806</td>\n",
       "      <td>0.279676</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.012286</td>\n",
       "      <td>0.996562</td>\n",
       "      <td>0.991595</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74257</th>\n",
       "      <td>0.346290</td>\n",
       "      <td>0.466003</td>\n",
       "      <td>0.897302</td>\n",
       "      <td>0.211842</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.342712</td>\n",
       "      <td>0.902577</td>\n",
       "      <td>0.564601</td>\n",
       "      <td>0.734670</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74258</th>\n",
       "      <td>0.156847</td>\n",
       "      <td>0.942993</td>\n",
       "      <td>0.999477</td>\n",
       "      <td>0.228354</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.998461</td>\n",
       "      <td>0.160573</td>\n",
       "      <td>0.987647</td>\n",
       "      <td>0.988458</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74259 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            dnn        rf      lgbm       ada  knn       mlp       svm  \\\n",
       "0      0.227917  0.991146  0.999975  0.288546  1.0  1.000000  0.091581   \n",
       "1      0.240160  0.991146  0.999979  0.288546  1.0  1.000000  0.027133   \n",
       "2      0.250123  0.991146  0.999971  0.278205  1.0  1.000000  0.910446   \n",
       "3      0.116905  0.563712  0.994721  0.233897  1.0  0.979222  0.001156   \n",
       "4      0.108086  0.828218  0.999755  0.295850  1.0  0.999907  0.967048   \n",
       "...         ...       ...       ...       ...  ...       ...       ...   \n",
       "74254  0.239538  0.991146  0.999975  0.288546  1.0  1.000000  0.014842   \n",
       "74255  0.161583  0.958180  0.999924  0.297158  1.0  1.000000  0.912447   \n",
       "74256  0.264320  0.974195  0.999806  0.279676  1.0  0.999999  0.012286   \n",
       "74257  0.346290  0.466003  0.897302  0.211842  0.6  0.342712  0.902577   \n",
       "74258  0.156847  0.942993  0.999477  0.228354  1.0  0.998461  0.160573   \n",
       "\n",
       "            cat       xgb  label  \n",
       "0      0.998578  0.996300    1.0  \n",
       "1      0.998578  0.996300    1.0  \n",
       "2      0.998199  0.996300    1.0  \n",
       "3      0.960457  0.951862    2.0  \n",
       "4      0.997023  0.977432    0.0  \n",
       "...         ...       ...    ...  \n",
       "74254  0.998425  0.996300    1.0  \n",
       "74255  0.997329  0.989533    1.0  \n",
       "74256  0.996562  0.991595    0.0  \n",
       "74257  0.564601  0.734670    0.0  \n",
       "74258  0.987647  0.988458    1.0  \n",
       "\n",
       "[74259 rows x 10 columns]"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74254</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74255</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74256</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74257</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74258</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74259 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       dnn   rf  lgbm  ada  knn  mlp  svm  cat  xgb  label\n",
       "0      0.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "1      0.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "2      1.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "3      1.0  2.0   2.0  2.0  2.0  2.0  0.0  2.0  2.0    2.0\n",
       "4      3.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "...    ...  ...   ...  ...  ...  ...  ...  ...  ...    ...\n",
       "74254  1.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "74255  1.0  1.0   1.0  2.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "74256  0.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "74257  0.0  0.0   0.0  0.0  0.0  3.0  0.0  0.0  0.0    0.0\n",
       "74258  0.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "\n",
       "[74259 rows x 10 columns]"
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_01 = df_level_01.pop('label')\n",
    "# X_01 = df_level_01\n",
    "# df_level_01 = df_level_01.assign(label = y_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_level_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split = 0.7\n",
    "\n",
    "# X_train_01,X_test_01, y_train_01, y_test_01 = sklearn.model_selection.train_test_split(X_01, y_01, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tabulate import tabulate\n",
    "\n",
    "# # Assuming data is a 110x4 list, where each row is a sublist\n",
    "# # data =  [[\"Row {} Col {}\".format(i + 1, j + 1) for j in range(4)] for i in range(110)]\n",
    "# data = [[\"\" for _ in range(3)] for _ in range(12)]\n",
    "\n",
    "# # Manually insert data at specific row and column\n",
    "# # data[0][0] = \"ADA\"\n",
    "# # data[1][0] = \"DNN\"\n",
    "# # data[2][0] = \"SVM\"\n",
    "# # data[3][0] = \"ADA\"\n",
    "# # data[4][0] = \"DNN\"\n",
    "# # data[2][0] = \"SVM\"\n",
    "\n",
    "\n",
    "# names_models = ['ADA',\n",
    "#                 'SVM',\n",
    "#                 'DNN',\n",
    "#                 'MLP',\n",
    "#                 'KNN',\n",
    "#                 'CAT',\n",
    "#                 'XGB',\n",
    "#                 'LGBM',\n",
    "#                 'RF',\n",
    "#                 'LR',\n",
    "#                 'VOTING'\n",
    "#                 ]\n",
    "# level_00_f1 = [ada_f1_00,\n",
    "#                 svm_f1_00,\n",
    "#                 dnn_f1_00,\n",
    "#                 mlp_f1_00,\n",
    "#                 knn_f1_00,\n",
    "#                 cat_f1_00,\n",
    "#                 xgb_f1_00,\n",
    "#                 lgbm_f1_00,\n",
    "#                 rf_f1_00,\n",
    "#                 lr_f1_00,\n",
    "#                 voting_f1_00]  \n",
    "\n",
    "                 \n",
    "\n",
    "# for i in range(0,len(names_models)):\n",
    "#     data[i][0] =  names_models[i]\n",
    "#     data[i][1] = level_00_f1[i]\n",
    "#     data[i][2] = level_01_f1[i]\n",
    "\n",
    "\n",
    " \n",
    "# # data[0][1] = ada_acc_00\n",
    "# # data\n",
    "\n",
    "# # Define column headers\n",
    "# headers = [\"F1\", \"Level 00\", \"Level 01\"]\n",
    "\n",
    "# # Print the table\n",
    "# table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "# print(table)\n",
    "# with open(output_file_name, \"a\") as f: print(table, file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_acc_00 = 0 \n",
    "voting_acc_00 = 0\n",
    "\n",
    "lr_pre_00 = 0 \n",
    "voting_pre_00 = 0\n",
    "\n",
    "lr_rec_00 = 0 \n",
    "voting_rec_00 = 0\n",
    "\n",
    "lr_f1_00 = 0 \n",
    "voting_f1_00 = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "| Models   | ACC-00             | PRE-00              | REC-00              | F1-00              |\n",
      "+==========+====================+=====================+=====================+====================+\n",
      "| ADA      | 0.7636650103017816 | 0.5202748124787877  | 0.5375625355482514  | 0.5199528225900233 |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "| SVM      | 0.8623466515843198 | 0.48425063000684077 | 0.46893233984628824 | 0.4740355799030905 |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "| DNN      | 0.625580737688361  | 0.25231843905556384 | 0.27694062367480204 | 0.2623953784372187 |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "| MLP      | 0.967142029922299  | 0.7209556541600184  | 0.7485466016871467  | 0.7339114879144161 |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "| KNN      | 0.9742253464226558 | 0.8414572813380907  | 0.7551588503706606  | 0.7527130001463106 |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "| CAT      | 0.9698487725393555 | 0.7353223075254655  | 0.7440350494273591  | 0.7396009841504471 |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "| XGB      | 0.9537295142676309 | 0.7135013155208901  | 0.7018953174003261  | 0.7072374594814604 |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "| LGBM     | 0.9800428230921504 | 0.7644634850194615  | 0.7727488923116625  | 0.7684486696844631 |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "| RF       | 0.9184206628152817 | 0.7228431075942422  | 0.5465452043853987  | 0.5501545271989657 |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "| LR       | 0                  | 0                   | 0                   | 0                  |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "| VOTING   | 0                  | 0                   | 0                   | 0                  |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n",
      "|          |                    |                     |                     |                    |\n",
      "+----------+--------------------+---------------------+---------------------+--------------------+\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "\n",
    "# Assuming data is a 110x4 list, where each row is a sublist\n",
    "# data =  [[\"Row {} Col {}\".format(i + 1, j + 1) for j in range(4)] for i in range(110)]\n",
    "data = [[\"\" for _ in range(5)] for _ in range(12)]\n",
    "\n",
    "# Manually insert data at specific row and column\n",
    "# data[0][0] = \"ADA\"\n",
    "# data[1][0] = \"DNN\"\n",
    "# data[2][0] = \"SVM\"\n",
    "# data[3][0] = \"ADA\"\n",
    "# data[4][0] = \"DNN\"\n",
    "# data[2][0] = \"SVM\"\n",
    "\n",
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'VOTING'\n",
    "                ]\n",
    "level_00_acc = [ada_acc_00,\n",
    "                svm_acc_00,\n",
    "                dnn_acc_00,\n",
    "                mlp_acc_00,\n",
    "                knn_acc_00,\n",
    "                cat_acc_00,\n",
    "                xgb_acc_00,\n",
    "                lgbm_acc_00,\n",
    "                rf_acc_00,\n",
    "                lr_acc_00,\n",
    "                voting_acc_00]  \n",
    "level_00_pre = [ada_pre_00,\n",
    "                svm_pre_00,\n",
    "                dnn_pre_00,\n",
    "                mlp_pre_00,\n",
    "                knn_pre_00,\n",
    "                cat_pre_00,\n",
    "                xgb_pre_00,\n",
    "                lgbm_pre_00,\n",
    "                rf_pre_00,\n",
    "                lr_pre_00,\n",
    "                voting_pre_00] \n",
    "level_00_rec = [ada_rec_00,\n",
    "                svm_rec_00,\n",
    "                dnn_rec_00,\n",
    "                mlp_rec_00,\n",
    "                knn_rec_00,\n",
    "                cat_rec_00,\n",
    "                xgb_rec_00,\n",
    "                lgbm_rec_00,\n",
    "                rf_rec_00,\n",
    "                lr_rec_00,\n",
    "                voting_rec_00]  \n",
    "level_00_f1 = [ada_f1_00,\n",
    "                svm_f1_00,\n",
    "                dnn_f1_00,\n",
    "                mlp_f1_00,\n",
    "                knn_f1_00,\n",
    "                cat_f1_00,\n",
    "                xgb_f1_00,\n",
    "                lgbm_f1_00,\n",
    "                rf_f1_00,\n",
    "                lr_f1_00,\n",
    "                voting_f1_00]  \n",
    "\n",
    "                 \n",
    "\n",
    "for i in range(0,len(names_models)):\n",
    "    data[i][0] =  names_models[i]\n",
    "\n",
    "    data[i][1] = level_00_acc[i]\n",
    "    # data[i][2] = level_01_acc[i]\n",
    "\n",
    "    data[i][2] = level_00_pre[i] \n",
    "    # data[i][4] = level_01_pre[i]\n",
    "\n",
    "    data[i][3] = level_00_rec[i] \n",
    "    # data[i][6] = level_01_rec[i]\n",
    "\n",
    "    data[i][4] = level_00_f1[i]\n",
    "    # data[i][8] = level_01_f1[i]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    " \n",
    "# data[0][1] = ada_acc_00\n",
    "# data\n",
    "\n",
    "# Define column headers\n",
    "# headers = [\"Models\", \"ACC-00\", \" ACC-01\",\"PRE-00\", \" PRE-01\",\"REC-00\", \" REC-01\",\"F1-00\", \" F1-01\",]\n",
    "headers = [\"Models\", \"ACC-00\",\"PRE-00\",\"REC-00\",\"F1-00\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
